{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "f0b82bb8-48fb-445a-a3fe-a09d1409e69f",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "from collections import Counter\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "4e4360f6-1aad-4979-8b9b-f1fc8e26abb4",
   "metadata": {},
   "outputs": [],
   "source": [
    "#############################################\n",
    "# 2. 准备文本\n",
    "#############################################\n",
    "# corpus = [\n",
    "#     \"I love programming\",\n",
    "#     \"Programming is fun\",\n",
    "#     \"I love machine  learning\"\n",
    "# ]\n",
    "corpus = [\n",
    "    \"我喜欢编程\",\n",
    "    \"编程很有趣\",\n",
    "    \"我喜欢机器学习\"\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "7c5693d0-b380-4c7f-884a-49c6cda9ea81",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[['我喜欢编程'], ['编程很有趣'], ['我喜欢机器学习']]\n"
     ]
    }
   ],
   "source": [
    "#############################################\n",
    "# 3. 数据预处理\n",
    "#############################################\n",
    "# 分词与小写化\n",
    "tokenized_corpus = [sentence.lower().split() for sentence in corpus]\n",
    "\n",
    "# 输出分词结果\n",
    "print(tokenized_corpus)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "e4cc9fc0-2c0a-4a34-9a0e-203f8bde4934",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'我喜欢编程': 0, '编程很有趣': 1, '我喜欢机器学习': 2}\n"
     ]
    }
   ],
   "source": [
    "#############################################\n",
    "# 4. 创建词表并进行编码\n",
    "#############################################\n",
    "# 利用Counter统计词表\n",
    "flat_list = [word for sentence in tokenized_corpus for word in sentence]\n",
    "word_counts = Counter(flat_list)\n",
    "vocab = {word:idx for idx, (word, _) in enumerate(word_counts.items())}\n",
    "\n",
    "#输出词汇表\n",
    "print(vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "627a63af-2555-4af6-a1fd-b48528449cd0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([ 0.5190,  1.5956,  1.0483,  1.5192,  0.1865, -0.4662, -0.7587,  1.1690,\n",
      "         0.9604, -0.5022], grad_fn=<EmbeddingBackward0>)\n"
     ]
    }
   ],
   "source": [
    "#############################################\n",
    "# 5. 使用嵌入词层将单词编码转换为向量\n",
    "#############################################\n",
    "\n",
    "# 设置嵌入维度\n",
    "embedding_dim = 10\n",
    "embedding = nn.Embedding(len(vocab), embedding_dim)\n",
    "\n",
    "# 输入单词的索引\n",
    "# example_word_idx = vocab[\"programming\"]\n",
    "example_word_idx = vocab[\"编程很有趣\"]\n",
    "word_vector = embedding(torch.tensor(example_word_idx))\n",
    "\n",
    "# 输出单词向量\n",
    "print(word_vector)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "fadb37bf-a61b-4f11-b2c1-7b0a9655d9e9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Word: '我喜欢编程' - Vector: tensor([ 0.9823, -0.5000, -2.7648, -0.8459, -1.4066, -0.6947,  1.1084,  0.8762,\n",
      "        -0.6614,  0.8607], grad_fn=<EmbeddingBackward0>)\n",
      "Word: '编程很有趣' - Vector: tensor([ 0.5190,  1.5956,  1.0483,  1.5192,  0.1865, -0.4662, -0.7587,  1.1690,\n",
      "         0.9604, -0.5022], grad_fn=<EmbeddingBackward0>)\n",
      "Word: '我喜欢机器学习' - Vector: tensor([ 1.7053, -1.0617,  1.3534,  0.8793, -1.8897, -0.5973,  0.4532, -0.5450,\n",
      "        -0.8747, -1.0170], grad_fn=<EmbeddingBackward0>)\n"
     ]
    }
   ],
   "source": [
    "#############################################\n",
    "# 6. 查看单词向量\n",
    "#############################################\n",
    "\n",
    "# 查看所有单词向量\n",
    "for word, idx in vocab.items():\n",
    "    print(f\"Word: '{word}' - Vector: {embedding(torch.tensor(idx))}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "1b131475-fb6e-4520-bc76-d9623e0f253d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Looking in indexes: https://pypi.tuna.tsinghua.edu.cn/simple/\n",
      "Collecting jieba\n",
      "  Downloading https://pypi.tuna.tsinghua.edu.cn/packages/c6/cb/18eeb235f833b726522d7ebed54f2278ce28ba9438e3135ab0278d9792a2/jieba-0.42.1.tar.gz (19.2 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m19.2/19.2 MB\u001b[0m \u001b[31m2.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25hBuilding wheels for collected packages: jieba\n",
      "  Building wheel for jieba (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for jieba: filename=jieba-0.42.1-py3-none-any.whl size=19314458 sha256=fe77269a92d191f955917cccaaaa18aee1207214a1e63e7834b3c6d302e94e8c\n",
      "  Stored in directory: /home/geobeans/.cache/pip/wheels/7b/3a/3e/1bf625b8dd63d53265aad527b244647c679dff9b60588a324f\n",
      "Successfully built jieba\n",
      "Installing collected packages: jieba\n",
      "Successfully installed jieba-0.42.1\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "pip install jieba"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "fb18848f-3291-4276-8e6f-f65bf586ba87",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['近日', '，', '重庆市', '一位', '业主', '连续', '几天', '发现', '家中', '天然气', '表', '异常', '抖动', '，', '因', '担心', '发生意外', '于是', '向', '重庆', '燃气', '集团', '反映', '问题', '。', '不料', '，', '被', '怼', '“', '死', '了', '国家', '会', '赔', '的', '，', '不要', '担心', '。', '”']\n"
     ]
    }
   ],
   "source": [
    "import jieba\n",
    "\n",
    "sentence = \"近日，重庆市一位业主连续几天发现家中天然气表异常抖动，因担心发生意外于是向重庆燃气集团反映问题。不料，被怼“死了国家会赔的，不要担心。”\"\n",
    "words = jieba.cut(sentence, cut_all=False)  # 精确模式\n",
    "print(list(words))\n",
    "\n",
    "# 输出：['我', '爱', '自然语言处理']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "20d77d77-554b-4732-8818-f8cddeed54a8",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "py-3.12_torch_24",
   "language": "python",
   "name": "py3.12_torch24"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
